package ca.uwindsor.cs.deepweb.utility.documentanalysis;


import java.io.IOException;

import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.standard.StandardAnalyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.index.CorruptIndexException;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.TermFreqVector;

import ca.uwindsor.cs.deepweb.estimation.FilesContentIndexer;

public class LuceneIndexTermsAnalyzer {
	
	public static final String FIELD_UNIQUE = "UNIQUE";
	
	public static final String FIELD_TOTAL = "TOTAL";

	/**
	 * @param args
	 */
	public static void main(String[] args) {
		// TODO Auto-generated method stub
//		args = new String[2];
//		args[0] = "F:\\indexes\\newsgroup_grouped";
//		args[1] = "F:\\DocumentNumberofUniqueTotalTermsIndex\\newsgroup_grouped";

		// TODO Auto-generated method stub
		if (args.length != 2) {
			System.out
					.println("java -jar LuceneIndexReader.jar indexDir outputDir");
			System.exit(0);
		}
		try {
			IndexReader reader = IndexReader.open(args[0]);
			int j;
			String id;
			Document doc;
			int unique;
			int total = 0;
			IndexWriter indexWriter;
			Analyzer luceneAnalyzer = new StandardAnalyzer();
			indexWriter = new IndexWriter(args[1], luceneAnalyzer, true);
			int[] frequences;
			
			for (int i = 0; i < reader.numDocs(); i++) {
				// 对contents的token之后的term存于了TermFreqVector
				doc = reader.document(i);
				id = doc.get(FilesContentIndexer.FIELD_ID);
				
				TermFreqVector termFreqVector = reader.getTermFreqVector(i,
						FilesContentIndexer.FIELD_CONTENT);

				if (termFreqVector == null) {
					System.out.println("termFreqVector is null.");
					continue;
				}

				unique = termFreqVector.getTerms().length;

				frequences = termFreqVector.getTermFrequencies();
				for (j = 0; j < frequences.length; j++) {
					total += frequences[j];
				}
				
				Document document = new Document();
				Field FieldID = new Field(FilesContentIndexer.FIELD_ID,
						id, Field.Store.YES, Field.Index.UN_TOKENIZED);

				Field FieldUnique = new Field(FIELD_UNIQUE ,
						Integer.toString(unique),
						Field.Store.YES, Field.Index.UN_TOKENIZED);
				
				Field FieldTotal = new Field(FIELD_TOTAL ,
						Integer.toString(total),
						Field.Store.YES, Field.Index.UN_TOKENIZED);

				document.add(FieldID);
				document.add(FieldUnique);
				document.add(FieldTotal);
				indexWriter.addDocument(document);
				// System.out.println("FieldName:" + fieldName);

			}
			//
			indexWriter.optimize();
			indexWriter.close();
			System.gc();
		} catch (CorruptIndexException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
			System.exit(-1);
		} catch (IOException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
			System.exit(-1);
		}
		System.out.println("finished");
	}

}
